{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "01d9266b-8be7-4f77-b26a-6a9c50b511a2",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:42:31.695281Z",
     "shell.execute_reply.started": "2023-09-27T06:41:52.029361Z",
     "to_execute": "2023-09-27T06:41:51.929Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>domain_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>3202332347979</td>\n",
       "      <td>2.088002e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>3202332347979</td>\n",
       "      <td>2.088002e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>3202332347979</td>\n",
       "      <td>2.088002e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>28</td>\n",
       "      <td>3202332347979</td>\n",
       "      <td>2.088002e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>495</td>\n",
       "      <td>3202332809370</td>\n",
       "      <td>2.088002e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76804211</th>\n",
       "      <td>76804211</td>\n",
       "      <td>28</td>\n",
       "      <td>3202332044786</td>\n",
       "      <td>2.088041e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76804212</th>\n",
       "      <td>76804212</td>\n",
       "      <td>28</td>\n",
       "      <td>3202332044786</td>\n",
       "      <td>2.088041e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76804213</th>\n",
       "      <td>76804213</td>\n",
       "      <td>27</td>\n",
       "      <td>3202332267723</td>\n",
       "      <td>2.088041e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76804214</th>\n",
       "      <td>76804214</td>\n",
       "      <td>27</td>\n",
       "      <td>3202332911449</td>\n",
       "      <td>2.088042e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76804215</th>\n",
       "      <td>76804215</td>\n",
       "      <td>28</td>\n",
       "      <td>3202333320018</td>\n",
       "      <td>2.088042e+15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76804216 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Unnamed: 0  domain_id        item_id       user_id\n",
       "0                  0         28  3202332347979  2.088002e+15\n",
       "1                  1         28  3202332347979  2.088002e+15\n",
       "2                  2         28  3202332347979  2.088002e+15\n",
       "3                  3         28  3202332347979  2.088002e+15\n",
       "4                  4        495  3202332809370  2.088002e+15\n",
       "...              ...        ...            ...           ...\n",
       "76804211    76804211         28  3202332044786  2.088041e+15\n",
       "76804212    76804212         28  3202332044786  2.088041e+15\n",
       "76804213    76804213         27  3202332267723  2.088041e+15\n",
       "76804214    76804214         27  3202332911449  2.088042e+15\n",
       "76804215    76804215         28  3202333320018  2.088042e+15\n",
       "\n",
       "[76804216 rows x 4 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "raw_data = pd.read_csv(\"/ossfs/workspace/CDSR/mybank_dataset/mybank_rawdata.csv\")\n",
    "raw_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "5bd7d3ba-1d9a-4f15-99ae-7995c84fdd81",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:43:50.513043Z",
     "shell.execute_reply.started": "2023-09-27T06:42:31.697839Z",
     "to_execute": "2023-09-27T06:41:52.025Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4792341 4729 52154519\n"
     ]
    }
   ],
   "source": [
    "# rate_size_dic_i = raw_data.groupby('item_id').size()\n",
    "# choosed_index_del_i = rate_size_dic_i.index[rate_size_dic_i < 10]\n",
    "# choosed_index_del_i = rate_size_dic_i.index[rate_size_dic_i < 10]\n",
    "# raw_data = raw_data[~raw_data['item_id'].isin(list(choosed_index_del_i))] # item freq more than 10\n",
    "rate_size_dic_u1 = raw_data.groupby('user_id').size()\n",
    "choosed_index_del_u1 = rate_size_dic_u1.index[rate_size_dic_u1 < 5]\n",
    "raw_data = raw_data[~raw_data['user_id'].isin(list(choosed_index_del_u1))]\n",
    "user_unique = list(raw_data['user_id'].unique())\n",
    "movie_unique = list(raw_data['item_id'].unique())\n",
    "u = len(user_unique)\n",
    "i = len(movie_unique)\n",
    "rating_num = len(raw_data)\n",
    "# ratings = raw_data.drop(columns=['cvr'])\n",
    "print(u,i,rating_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "4474cd9a-cd5f-4cfc-85fd-b30ed35983d2",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:46:08.517043Z",
     "shell.execute_reply.started": "2023-09-27T06:43:50.514771Z",
     "to_execute": "2023-09-27T06:41:52.063Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all user number :4792341, item number :4729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    }
   ],
   "source": [
    "ratings = raw_data\n",
    "user_unique1 = list(ratings['user_id'].unique())\n",
    "item_unique1 = list(ratings['item_id'].unique())\n",
    "user_dict = dict()\n",
    "item_dict = dict()\n",
    "for i in range(len(user_unique1)):\n",
    "    user_dict[user_unique1[i]] = i\n",
    "for i in range(len(item_unique1)):\n",
    "    item_dict[item_unique1[i]] = i\n",
    "ratings['user_id'] = ratings['user_id'].apply(lambda x :user_dict[x])\n",
    "ratings['item_id'] = ratings['item_id'].apply(lambda x :item_dict[x])\n",
    "print(\"all user number :{}, item number :{}\".format(len(user_dict),len(item_dict)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "8b59c786-d85d-46e1-b292-a4b86c40a641",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:46:19.011973Z",
     "shell.execute_reply.started": "2023-09-27T06:46:08.524090Z",
     "to_execute": "2023-09-27T06:41:52.093Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22819115 15553529 13781875\n"
     ]
    }
   ],
   "source": [
    "data_d1 = ratings[ratings['domain_id'] == 27]\n",
    "data_d1 = data_d1.loc[:, ['item_id','user_id']]\n",
    "\n",
    "data_d2 = ratings[ratings['domain_id'] == 28]\n",
    "data_d2 = data_d2.loc[:, ['item_id','user_id']]\n",
    "\n",
    "data_d3 = ratings[ratings['domain_id'] == 495]\n",
    "data_d3 = data_d3.loc[:, ['item_id','user_id']]\n",
    "print(len(data_d1),len(data_d2),len(data_d3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "72726454-d4c6-4a1d-8ca3-a87cfcf3d2d2",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:47:56.546883Z",
     "shell.execute_reply.started": "2023-09-27T06:46:19.014519Z",
     "to_execute": "2023-09-27T06:41:52.117Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def find_dict(ratings):\n",
    "    seq = defaultdict(list)  \n",
    "    uid = ratings['user_id'].tolist()\n",
    "    iid = ratings['item_id'].tolist()\n",
    "    for i in range(len(uid)):\n",
    "        seq[uid[i]].append(iid[i])\n",
    "    return seq\n",
    "\n",
    "seq1 = find_dict(data_d1)\n",
    "seq2 = find_dict(data_d2)\n",
    "seq3 = find_dict(data_d3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "f218a534-8e69-431b-9629-7f850561370f",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:47:58.341136Z",
     "shell.execute_reply.started": "2023-09-27T06:47:56.548857Z",
     "to_execute": "2023-09-27T06:41:52.139Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [],
   "source": [
    "# another dataset\n",
    "data_d2 = data_d3\n",
    "seq2 = seq3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "7ad33a6f-8d21-45e3-b4c7-c829f451ace2",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:48:38.015777Z",
     "shell.execute_reply.started": "2023-09-27T06:47:58.344223Z",
     "to_execute": "2023-09-27T06:41:52.169Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [],
   "source": [
    "user_unique1 = list(data_d1['user_id'].unique())\n",
    "user_unique2 = list(data_d2['user_id'].unique())\n",
    "user_node,seq_d1, seq_d2, domain_id  = [], [], [], []\n",
    "\n",
    "for u_id_tmp in user_unique1:\n",
    "    if len(seq1[u_id_tmp])>=5 and (len(seq2[u_id_tmp])>=5 or len(seq2[u_id_tmp])==0):\n",
    "        user_node.append(u_id_tmp)\n",
    "        seq_d1.append(seq1[u_id_tmp])\n",
    "        seq_d2.append(seq2[u_id_tmp])\n",
    "        domain_id.append(0)\n",
    "\n",
    "for u_id_tmp in user_unique2:\n",
    "    if len(seq2[u_id_tmp])>=5 and (len(seq1[u_id_tmp])>=5 or len(seq1[u_id_tmp])==0):\n",
    "        user_node.append(u_id_tmp)\n",
    "        seq_d1.append(seq1[u_id_tmp])\n",
    "        seq_d2.append(seq2[u_id_tmp])\n",
    "        domain_id.append(1)\n",
    "\n",
    "df_all = pd.DataFrame({'user_id':user_node,'seq_d1':seq_d1,'seq_d2':seq_d2,'domain_id':domain_id})\n",
    "df_all['seq_d1'] = df_all['seq_d1'].apply(lambda x: list(set(x)))\n",
    "df_all['seq_d2'] = df_all['seq_d2'].apply(lambda x: list(set(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "e4a183e0-6076-4a30-8bb2-0818b08413b3",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:48:43.612442Z",
     "shell.execute_reply.started": "2023-09-27T06:48:38.017201Z",
     "to_execute": "2023-09-27T06:41:52.205Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2436138\n",
      "531728\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>seq_d1</th>\n",
       "      <th>seq_d2</th>\n",
       "      <th>domain_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>32</td>\n",
       "      <td>[581, 231, 1256, 45, 1267, 19, 630, 378]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>69</td>\n",
       "      <td>[97, 34, 18, 53, 85, 30]</td>\n",
       "      <td>[1601, 2, 102, 167, 104, 271, 306]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>83</td>\n",
       "      <td>[36, 38, 1267, 85, 93, 607]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>126</td>\n",
       "      <td>[34, 581, 357, 38, 231, 82, 19, 85, 93, 31]</td>\n",
       "      <td>[7, 10, 586, 23, 637]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>140</td>\n",
       "      <td>[34, 581, 168, 364, 45, 78, 18, 1587]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531723</th>\n",
       "      <td>4774827</td>\n",
       "      <td>[]</td>\n",
       "      <td>[579, 454, 838, 589, 25]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531724</th>\n",
       "      <td>4777579</td>\n",
       "      <td>[]</td>\n",
       "      <td>[352, 1, 746, 1629, 287]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531725</th>\n",
       "      <td>312177</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1632, 367, 272, 1909, 287]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531726</th>\n",
       "      <td>4784743</td>\n",
       "      <td>[]</td>\n",
       "      <td>[682, 206, 1552, 3223, 2142]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531727</th>\n",
       "      <td>4785697</td>\n",
       "      <td>[]</td>\n",
       "      <td>[352, 838, 40, 300, 47, 284]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>531728 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                                       seq_d1  \\\n",
       "0            32     [581, 231, 1256, 45, 1267, 19, 630, 378]   \n",
       "1            69                     [97, 34, 18, 53, 85, 30]   \n",
       "2            83                  [36, 38, 1267, 85, 93, 607]   \n",
       "3           126  [34, 581, 357, 38, 231, 82, 19, 85, 93, 31]   \n",
       "4           140        [34, 581, 168, 364, 45, 78, 18, 1587]   \n",
       "...         ...                                          ...   \n",
       "531723  4774827                                           []   \n",
       "531724  4777579                                           []   \n",
       "531725   312177                                           []   \n",
       "531726  4784743                                           []   \n",
       "531727  4785697                                           []   \n",
       "\n",
       "                                    seq_d2  domain_id  \n",
       "0                                       []          0  \n",
       "1       [1601, 2, 102, 167, 104, 271, 306]          0  \n",
       "2                                       []          0  \n",
       "3                    [7, 10, 586, 23, 637]          0  \n",
       "4                                       []          0  \n",
       "...                                    ...        ...  \n",
       "531723            [579, 454, 838, 589, 25]          1  \n",
       "531724            [352, 1, 746, 1629, 287]          1  \n",
       "531725         [1632, 367, 272, 1909, 287]          1  \n",
       "531726        [682, 206, 1552, 3223, 2142]          1  \n",
       "531727        [352, 838, 40, 300, 47, 284]          1  \n",
       "\n",
       "[531728 rows x 4 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_all))\n",
    "user_node_new = df_all['user_id'].tolist()\n",
    "seq_d1_new = df_all['seq_d1'].tolist()\n",
    "seq_d2_new = df_all['seq_d2'].tolist()\n",
    "domain_id_new = df_all['domain_id'].tolist()\n",
    "user_node,seq_d1, seq_d2, domain_id  = [], [], [], []\n",
    "for i in range(len(user_node_new)):\n",
    "    user_node_tmp = user_node_new[i]\n",
    "    seq_d1_tmp = seq_d1_new[i]\n",
    "    seq_d2_tmp = seq_d2_new[i]\n",
    "    domain_id_tmp = domain_id_new[i]\n",
    "    if (len(seq_d1_tmp)==0 or len(seq_d1_tmp)>=5) and (len(seq_d2_tmp)==0 or len(seq_d2_tmp)>=5) :\n",
    "        user_node.append(user_node_tmp)\n",
    "        seq_d1.append(seq_d1_tmp)\n",
    "        seq_d2.append(seq_d2_tmp)\n",
    "        domain_id.append(domain_id_tmp)\n",
    "df_all_new = pd.DataFrame({'user_id':user_node,'seq_d1':seq_d1,'seq_d2':seq_d2,'domain_id':domain_id})\n",
    "print(len(df_all_new))\n",
    "df_all_new\n",
    "# data = df_all.sample(frac=1.0).sample(n=100000, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "11e64913-2210-4d4e-9e96-f3fa0c7053d9",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:48:47.903527Z",
     "shell.execute_reply.started": "2023-09-27T06:48:43.614986Z",
     "to_execute": "2023-09-27T06:41:52.234Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [],
   "source": [
    "df_all_new.to_csv(\"/ossfs/workspace/CDSR/mybank_dataset/mybank_filter_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "d13128c3-d6d3-485d-a48c-3ce924b9dcc7",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:48:49.635532Z",
     "shell.execute_reply.started": "2023-09-27T06:48:47.908160Z",
     "to_execute": "2023-09-27T06:41:52.270Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2602 767\n"
     ]
    }
   ],
   "source": [
    "# larger than 999\n",
    "from itertools import chain\n",
    "print(len(list(set(list(chain(*list(df_all_new['seq_d1'])))))),len(list(set(list(chain(*list(df_all_new['seq_d2'])))))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "3ea0e566-c863-4de6-814a-b6b364309428",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:49:14.802167Z",
     "shell.execute_reply.started": "2023-09-27T06:48:49.637960Z",
     "to_execute": "2023-09-27T06:41:52.300Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3247 915\n"
     ]
    }
   ],
   "source": [
    "# random item set \n",
    "import pandas as pd\n",
    "import json\n",
    "data = df_all_new\n",
    "import random\n",
    "\n",
    "def process_list(my_list):\n",
    "    # 百分之百的概率，随机弹出两个元素\n",
    "    if len(my_list) >= 2:\n",
    "        popped_items = random.sample(my_list, 2)\n",
    "        my_list = [item for item in my_list if item not in popped_items]\n",
    "    else:\n",
    "        popped_items = []\n",
    "\n",
    "    # 百分之五十的概率，给每个元素加一个1-1000的随机整数\n",
    "    for i in range(len(my_list)):\n",
    "        if random.random() < 0.5:\n",
    "            my_list[i] += random.randint(1, 1000)\n",
    "    \n",
    "    random.shuffle(my_list)\n",
    "    \n",
    "    return my_list\n",
    "    \n",
    "# data = pd.read_csv(\"/ossfs/workspace/CDSR/mybank_dataset/mybank_filter_data.csv\")\n",
    "item_unique_d1 = list(data_d1['item_id'].unique())\n",
    "item_d1_start = 10000 \n",
    "item_unique_d2 = list(data_d2['item_id'].unique())\n",
    "item_d2_start = 1000000\n",
    "print(len(item_unique_d1),len(item_unique_d2))\n",
    "user_node_new = data['user_id'].tolist()\n",
    "seq_d1_new = data['seq_d1'].tolist()\n",
    "seq_d2_new = data['seq_d2'].tolist()\n",
    "domain_id_new = data['domain_id'].tolist()\n",
    "seq_d1_random, seq_d2_random  = [], []\n",
    "zs1, zs2 = 0, 0\n",
    "for i in range(len(user_node_new)):\n",
    "    seq_d1_tmp = seq_d1_new[i]\n",
    "    seq_d2_tmp = seq_d2_new[i]\n",
    "    if len(seq_d1_tmp)!=0:\n",
    "        zs1 += 1\n",
    "        seq_d1_tmp = [item_d1_start] + seq_d1_tmp\n",
    "        seq_d1_tmp = process_list(seq_d1_tmp)\n",
    "        if zs1 % 10 ==0:\n",
    "            item_d1_start += 1\n",
    "            \n",
    "    if len(seq_d2_tmp)!=0:\n",
    "        zs2 += 1\n",
    "        seq_d2_tmp = [item_d2_start] + seq_d2_tmp\n",
    "        seq_d2_tmp = process_list(seq_d2_tmp)\n",
    "        if zs2 % 10 ==0:\n",
    "            item_d2_start += 1\n",
    "    seq_d1_random.append(seq_d1_tmp)\n",
    "    seq_d2_random.append(seq_d2_tmp)\n",
    "seq_d1_random2, seq_d2_random2  = [], []\n",
    "for i in range(len(user_node_new)):\n",
    "    seq_d1_tmp = seq_d1_random[i]\n",
    "    seq_d2_tmp = seq_d2_random[i]\n",
    "    if len(seq_d1_tmp)!=0:\n",
    "        zs1 += 1\n",
    "        seq_d1_tmp = [item_d1_start] + seq_d1_tmp\n",
    "        if zs1 % 10 ==0:\n",
    "            item_d1_start += 1\n",
    "            \n",
    "    if len(seq_d2_tmp)!=0:\n",
    "        zs2 += 1\n",
    "        seq_d2_tmp = [item_d2_start] + seq_d2_tmp\n",
    "        if zs2 % 10 ==0:\n",
    "            item_d2_start += 1\n",
    "    seq_d1_random2.append(seq_d1_tmp)\n",
    "    seq_d2_random2.append(seq_d2_tmp)\n",
    "df_all = pd.DataFrame({'user_id':user_node_new,'seq_d1':seq_d1_random2,'seq_d2':seq_d2_random2,'domain_id':domain_id_new})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "fcebc5bb-4950-4b25-b0de-314e33abbe08",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:49:15.424931Z",
     "shell.execute_reply.started": "2023-09-27T06:49:14.803582Z",
     "to_execute": "2023-09-27T06:41:52.336Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58602 68435\n"
     ]
    }
   ],
   "source": [
    "# print(item_d1_start,item_d2_start)\n",
    "print(len(list(set(list(chain(*list(df_all['seq_d1'])))))),len(list(set(list(chain(*list(df_all['seq_d2'])))))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "5f79a244-ae56-4141-99a6-168199ba8976",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:49:22.634856Z",
     "shell.execute_reply.started": "2023-09-27T06:49:15.427308Z",
     "to_execute": "2023-09-27T06:41:52.365Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all user number :501173, item number :123133\n"
     ]
    }
   ],
   "source": [
    "from itertools import chain\n",
    "user_unique1 = list(df_all['user_id'].unique())\n",
    "item_unique = list(df_all['seq_d1']) + list(df_all['seq_d2'])\n",
    "item_unique1 = list(set(list(chain(*item_unique))))\n",
    "user_dict = dict()\n",
    "item_dict = dict()\n",
    "for i in range(len(user_unique1)):\n",
    "    user_dict[user_unique1[i]] = i\n",
    "for i in range(len(item_unique1)):\n",
    "    item_dict[item_unique1[i]] = i\n",
    "df_all['user_id'] = df_all['user_id'].apply(lambda x :user_dict[x])\n",
    "def get_item_hash(item):\n",
    "    return [item_dict[i] for i in item]\n",
    "df_all['seq_d1'] = df_all['seq_d1'].apply(get_item_hash)\n",
    "df_all['seq_d2'] = df_all['seq_d2'].apply(get_item_hash)\n",
    "print(\"all user number :{}, item number :{}\".format(len(user_dict),len(item_dict)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "74a7aa46-b160-45e7-a57e-b0401ae02a2b",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:49:22.940725Z",
     "shell.execute_reply.started": "2023-09-27T06:49:22.636458Z",
     "to_execute": "2023-09-27T06:41:52.399Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47346\n",
      "55679\n"
     ]
    }
   ],
   "source": [
    "# print(df_all)\n",
    "df_all = df_all.sample(frac=1.0).sample(n=100000, random_state=42)\n",
    "print(len(list(set(list(chain(*list(df_all['seq_d1'])))))))\n",
    "print(len(list(set(list(chain(*list(df_all['seq_d2'])))))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "96e3b084-bef2-40cc-9d44-18e0dc934cac",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:49:24.313487Z",
     "shell.execute_reply.started": "2023-09-27T06:49:22.943047Z",
     "to_execute": "2023-09-27T06:41:52.431Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [],
   "source": [
    "data = df_all.sample(frac=1.0)\n",
    "train_len = int(data.shape[0] * 0.80)\n",
    "save_data_train = data.iloc[ : train_len]\n",
    "save_data_val = data.iloc[ train_len: ]\n",
    "train_name = \"/ossfs/workspace/CDSR/mybank_dataset/loan_account_train100.csv\"\n",
    "val_name = \"/ossfs/workspace/CDSR/mybank_dataset/loan_account_test.csv\"\n",
    "save_data_train.to_csv(train_name, index=False)\n",
    "save_data_val.to_csv(val_name, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "bf51d0e2-a795-498e-8ec8-8cecdc640b26",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2023-09-27T06:49:29.170729Z",
     "shell.execute_reply.started": "2023-09-27T06:49:24.315839Z",
     "to_execute": "2023-09-27T06:41:52.470Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9345 70655\n",
      "17663\n",
      "9345 70655\n",
      "52991\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import random\n",
    "from typing import DefaultDict\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "import json\n",
    "from random import sample\n",
    "\n",
    "def select_overlap_user(train_name,save_train_name,overlap_ratio):\n",
    "    data = pd.read_csv(train_name)\n",
    "    user_node = data['user_id'].tolist()\n",
    "    seq_d1 = data['seq_d1'].tolist()\n",
    "    seq_d2 = data['seq_d2'].tolist()\n",
    "    domain_id = data['domain_id'].tolist()\n",
    "    user_node_overlap,seq_d1_overlap, seq_d2_overlap, domain_id_overlap  = [], [], [], []\n",
    "    user_node_nolap,seq_d1_nolap, seq_d2_nolap, domain_id_nolap  = [], [], [], []\n",
    "    for i in range(len(user_node)):\n",
    "        seq1_tmp = json.loads(seq_d1[i])\n",
    "        seq2_tmp = json.loads(seq_d2[i])\n",
    "        if len(seq1_tmp)!=0 and len(seq2_tmp)!=0:\n",
    "            user_node_overlap.append(user_node[i])\n",
    "            seq_d1_overlap.append(seq1_tmp)\n",
    "            seq_d2_overlap.append(seq2_tmp)\n",
    "            domain_id_overlap.append(domain_id[i])\n",
    "        else :\n",
    "            user_node_nolap.append(user_node[i])\n",
    "            seq_d1_nolap.append(seq1_tmp)\n",
    "            seq_d2_nolap.append(seq2_tmp)\n",
    "            domain_id_nolap.append(domain_id[i])\n",
    "    print(len(user_node_overlap),len(user_node_nolap)) # 3384 69945\n",
    "    #nolap_num = int(len(user_node_overlap)/overlap_ratio-len(user_node_overlap)) # 3384 + \n",
    "    sample_nolap_num = int(len(user_node_nolap)*overlap_ratio)\n",
    "    idx_lst = [i for i in range(len(user_node_nolap))]\n",
    "    select_idx = sample(idx_lst, sample_nolap_num)\n",
    "    print(sample_nolap_num)\n",
    "    # print(select_idx)\n",
    "    for idx_tmp in select_idx:\n",
    "        user_node_overlap.append(user_node_nolap[idx_tmp])\n",
    "        seq_d1_overlap.append(seq_d1_nolap[idx_tmp])\n",
    "        seq_d2_overlap.append(seq_d2_nolap[idx_tmp])\n",
    "        domain_id_overlap.append(domain_id_nolap[idx_tmp])\n",
    "    dataframe = pd.DataFrame({'user_id':user_node_overlap,'seq_d1':seq_d1_overlap,'seq_d2':seq_d2_overlap,'domain_id':domain_id_overlap})\n",
    "    dataframe.to_csv(save_train_name,index=False,sep=',')\n",
    "\n",
    "train_name = \"/ossfs/workspace/CDSR/mybank_dataset/loan_account_train100.csv\"\n",
    "\n",
    "ratios = [0.25,0.75]\n",
    "for overlap_ratio in ratios:\n",
    "    save_train_name = \"/ossfs/workspace/CDSR/mybank_dataset/loan_account_train\"+str(int(overlap_ratio*100))+\".csv\"\n",
    "    select_overlap_user(train_name,save_train_name,overlap_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "10c9a527-e660-4ba9-b45b-6e805592f4e9",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "",
     "shell.execute_reply.started": "2023-09-27T06:49:29.172592Z",
     "to_execute": "2023-09-27T06:41:52.517Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9345 70655\n",
      "17663 52992\n",
      "9345 70655\n",
      "52991 17664\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import random\n",
    "from typing import DefaultDict\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "import json\n",
    "from random import sample\n",
    "\n",
    "def select_overlap_user(train_name,save_train_name,overlap_ratio):\n",
    "    data = pd.read_csv(train_name)\n",
    "    user_node = data['user_id'].tolist()\n",
    "    seq_d1 = data['seq_d1'].tolist()\n",
    "    seq_d2 = data['seq_d2'].tolist()\n",
    "    domain_id = data['domain_id'].tolist()\n",
    "    user_node_overlap,seq_d1_overlap, seq_d2_overlap, domain_id_overlap  = [], [], [], []\n",
    "    user_node_nolap,seq_d1_nolap, seq_d2_nolap, domain_id_nolap, ob_nolap  = [], [], [], [], []\n",
    "    for i in range(len(user_node)):\n",
    "        seq1_tmp = json.loads(seq_d1[i])\n",
    "        seq2_tmp = json.loads(seq_d2[i])\n",
    "        if len(seq1_tmp)!=0 and len(seq2_tmp)!=0:\n",
    "            user_node_overlap.append(user_node[i])\n",
    "            seq_d1_overlap.append(seq1_tmp)\n",
    "            seq_d2_overlap.append(seq2_tmp)\n",
    "            domain_id_overlap.append(domain_id[i])\n",
    "            ob_nolap.append(1)\n",
    "        else :\n",
    "            user_node_nolap.append(user_node[i])\n",
    "            seq_d1_nolap.append(seq1_tmp)\n",
    "            seq_d2_nolap.append(seq2_tmp)\n",
    "            domain_id_nolap.append(domain_id[i])\n",
    "            # ob_nolap.append(1) # observed data\n",
    "    print(len(user_node_overlap),len(user_node_nolap)) # 3384 69945\n",
    "    #nolap_num = int(len(user_node_overlap)/overlap_ratio-len(user_node_overlap)) # 3384 + \n",
    "    sample_nolap_num = int(len(user_node_nolap)*overlap_ratio)\n",
    "    idx_lst = [i for i in range(len(user_node_nolap))]\n",
    "    select_idx = sample(idx_lst, sample_nolap_num)\n",
    "    not_select_idx = list(set(idx_lst).difference(set(select_idx)))\n",
    "    print(sample_nolap_num,len(not_select_idx))\n",
    "    # print(select_idx)\n",
    "    for idx_tmp in select_idx:\n",
    "        user_node_overlap.append(user_node_nolap[idx_tmp])\n",
    "        seq_d1_overlap.append(seq_d1_nolap[idx_tmp])\n",
    "        seq_d2_overlap.append(seq_d2_nolap[idx_tmp])\n",
    "        domain_id_overlap.append(domain_id_nolap[idx_tmp])\n",
    "        # user_node_nolap.append(user_node_overlap[idx_tmp])\n",
    "        # seq_d1_nolap.append(seq_d1_overlap[idx_tmp])\n",
    "        # seq_d2_nolap.append(seq_d2_overlap[idx_tmp])\n",
    "        # domain_id_nolap.append(domain_id_overlap[idx_tmp])\n",
    "        ob_nolap.append(1) # observed data\n",
    "    # print(len(user_node_nolap))\n",
    "    for idx_tmp in not_select_idx:\n",
    "        user_node_overlap.append(user_node_nolap[idx_tmp])\n",
    "        seq_d1_overlap.append(seq_d1_nolap[idx_tmp])\n",
    "        seq_d2_overlap.append(seq_d2_nolap[idx_tmp])\n",
    "        domain_id_overlap.append(domain_id_nolap[idx_tmp])\n",
    "        # user_node_nolap.append(user_node_overlap[idx_tmp])\n",
    "        # seq_d1_nolap.append(seq_d1_overlap[idx_tmp])\n",
    "        # seq_d2_nolap.append(seq_d2_overlap[idx_tmp])\n",
    "        # domain_id_nolap.append(domain_id_overlap[idx_tmp])\n",
    "        ob_nolap.append(0) # observed data\n",
    "    # append not observed data \n",
    "    dataframe = pd.DataFrame({'user_id':user_node_overlap,'seq_d1':seq_d1_overlap,'seq_d2':seq_d2_overlap,'domain_id':domain_id_overlap,'ob_label':ob_nolap})\n",
    "    dataframe.to_csv(save_train_name,index=False,sep=',')\n",
    "\n",
    "train_name = \"/ossfs/workspace/CDSR/mybank_dataset/loan_account_train100.csv\"\n",
    "\n",
    "ratios = [0.25,0.75]\n",
    "for overlap_ratio in ratios:\n",
    "    save_train_name = \"/ossfs/workspace/CDSR/mybank_dataset/loan_account_train\"+str(int(overlap_ratio*100))+\"_DR.csv\"\n",
    "    select_overlap_user(train_name,save_train_name,overlap_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eb77d8b9-46b8-4c8e-93ed-144bb7a0b258",
   "metadata": {
    "execution": {
     "shell.execute_reply.end": "2024-02-26T03:06:55.441993Z",
     "shell.execute_reply.started": "2024-02-26T03:06:55.198390Z",
     "to_execute": "2024-02-26T03:06:55.096Z"
    },
    "libroFormatter": "formatter-string"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62336\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"/ossfs/workspace/CDSR/mybank_dataset/loan_account_train75.csv\")\n",
    "print(len(df))"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 5
}
